{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Proximal Policy Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "gym.logger.set_level(40)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim \n",
    "from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from timeit import default_timer as timer\n",
    "from datetime import timedelta\n",
    "import os\n",
    "import glob\n",
    "\n",
    "\n",
    "from baselines.common.vec_env.dummy_vec_env import DummyVecEnv\n",
    "from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv\n",
    "\n",
    "from utils.hyperparameters import Config\n",
    "from utils.plot import plot\n",
    "from utils.wrappers import make_env_a2c_atari\n",
    "from agents.A2C import Model as A2C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = \"/tmp/gym_ppo/\"\n",
    "\n",
    "try:\n",
    "    os.makedirs(log_dir)\n",
    "except OSError:\n",
    "    files = glob.glob(os.path.join(log_dir, '*.monitor.csv'))\n",
    "    for f in files:\n",
    "        os.remove(f)\n",
    "\n",
    "config = Config()\n",
    "\n",
    "#ppo control\n",
    "config.ppo_epoch = 3\n",
    "config.num_mini_batch = 32\n",
    "config.ppo_clip_param = 0.1\n",
    "\n",
    "#a2c control\n",
    "config.num_agents=8\n",
    "config.rollout=128\n",
    "config.USE_GAE = True\n",
    "config.gae_tau = 0.95\n",
    "\n",
    "#misc agent variables\n",
    "config.GAMMA=0.99\n",
    "config.LR=7e-4\n",
    "config.entropy_loss_weight=0.01\n",
    "config.value_loss_weight=1.0\n",
    "config.grad_norm_max = 0.5\n",
    "\n",
    "config.MAX_FRAMES=int(1e7 / config.num_agents / config.rollout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rollout Storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RolloutStorage(object):\n",
    "    def __init__(self, num_steps, num_processes, obs_shape, action_space, device, USE_GAE=True, gae_tau=0.95):\n",
    "        self.observations = torch.zeros(num_steps + 1, num_processes, *obs_shape).to(device)\n",
    "        self.rewards = torch.zeros(num_steps, num_processes, 1).to(device)\n",
    "        self.value_preds = torch.zeros(num_steps + 1, num_processes, 1).to(device)\n",
    "        self.returns = torch.zeros(num_steps + 1, num_processes, 1).to(device)\n",
    "        self.action_log_probs = torch.zeros(num_steps, num_processes, 1).to(device)\n",
    "        self.actions = torch.zeros(num_steps, num_processes, 1).to(device, torch.long)\n",
    "        self.masks = torch.ones(num_steps + 1, num_processes, 1).to(device)\n",
    "\n",
    "        self.num_steps = num_steps\n",
    "        self.step = 0\n",
    "        self.gae = USE_GAE\n",
    "        self.gae_tau = gae_tau\n",
    "\n",
    "    def insert(self, current_obs, action, action_log_prob, value_pred, reward, mask):\n",
    "        self.observations[self.step + 1].copy_(current_obs)\n",
    "        self.actions[self.step].copy_(action)\n",
    "        self.action_log_probs[self.step].copy_(action_log_prob)\n",
    "        self.value_preds[self.step].copy_(value_pred)\n",
    "        self.rewards[self.step].copy_(reward)\n",
    "        self.masks[self.step + 1].copy_(mask)\n",
    "\n",
    "        self.step = (self.step + 1) % self.num_steps\n",
    "\n",
    "    def after_update(self):\n",
    "        self.observations[0].copy_(self.observations[-1])\n",
    "        self.masks[0].copy_(self.masks[-1])\n",
    "\n",
    "    def compute_returns(self, next_value, gamma):\n",
    "        if self.gae:\n",
    "            self.value_preds[-1] = next_value\n",
    "            gae = 0\n",
    "            for step in reversed(range(self.rewards.size(0))):\n",
    "                delta = self.rewards[step] + gamma * self.value_preds[step + 1] * self.masks[step + 1] - self.value_preds[step]\n",
    "                gae = delta + gamma * self.gae_tau * self.masks[step + 1] * gae\n",
    "                self.returns[step] = gae + self.value_preds[step]\n",
    "        else:\n",
    "            self.returns[-1] = next_value\n",
    "            for step in reversed(range(self.rewards.size(0))):\n",
    "                self.returns[step] = self.returns[step + 1] * \\\n",
    "                    gamma * self.masks[step + 1] + self.rewards[step]\n",
    "\n",
    "    def feed_forward_generator(self, advantages, num_mini_batch):\n",
    "        num_steps, num_processes = self.rewards.size()[0:2]\n",
    "        batch_size = num_processes * num_steps\n",
    "        assert batch_size >= num_mini_batch, (\n",
    "            f\"PPO requires the number processes ({num_processes}) \"\n",
    "            f\"* number of steps ({num_steps}) = {num_processes * num_steps} \"\n",
    "            f\"to be greater than or equal to the number of PPO mini batches ({num_mini_batch}).\")\n",
    "        mini_batch_size = batch_size // num_mini_batch\n",
    "        sampler = BatchSampler(SubsetRandomSampler(range(batch_size)), mini_batch_size, drop_last=False)\n",
    "        for indices in sampler:\n",
    "            observations_batch = self.observations[:-1].view(-1,\n",
    "                                        *self.observations.size()[2:])[indices]\n",
    "            actions_batch = self.actions.view(-1, self.actions.size(-1))[indices]\n",
    "            return_batch = self.returns[:-1].view(-1, 1)[indices]\n",
    "            masks_batch = self.masks[:-1].view(-1, 1)[indices]\n",
    "            old_action_log_probs_batch = self.action_log_probs.view(-1, 1)[indices]\n",
    "            adv_targ = advantages.view(-1, 1)[indices]\n",
    "\n",
    "            yield observations_batch, actions_batch, return_batch, masks_batch, old_action_log_probs_batch, adv_targ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(A2C):\n",
    "    def __init__(self, static_policy=False, env=None, config=None):\n",
    "        super(Model, self).__init__(static_policy, env, config)\n",
    "        \n",
    "        self.num_agents = config.num_agents\n",
    "        self.value_loss_weight = config.value_loss_weight\n",
    "        self.entropy_loss_weight = config.entropy_loss_weight\n",
    "        self.rollout = config.rollout\n",
    "        self.grad_norm_max = config.grad_norm_max\n",
    "\n",
    "        self.ppo_epoch = config.ppo_epoch\n",
    "        self.num_mini_batch = config.num_mini_batch\n",
    "        self.clip_param = config.ppo_clip_param\n",
    "\n",
    "        self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, eps=1e-5)\n",
    "        \n",
    "        self.rollouts = RolloutStorage(self.rollout, self.num_agents,\n",
    "            self.num_feats, self.env.action_space, self.device, config.USE_GAE, config.gae_tau)\n",
    "\n",
    "    def compute_loss(self, sample):\n",
    "        observations_batch, actions_batch, return_batch, masks_batch, old_action_log_probs_batch, adv_targ = sample\n",
    "\n",
    "        values, action_log_probs, dist_entropy = self.evaluate_actions(observations_batch, actions_batch)\n",
    "\n",
    "        ratio = torch.exp(action_log_probs - old_action_log_probs_batch)\n",
    "        surr1 = ratio * adv_targ\n",
    "        surr2 = torch.clamp(ratio, 1.0 - self.clip_param, 1.0 + self.clip_param) * adv_targ\n",
    "        action_loss = -torch.min(surr1, surr2).mean()\n",
    "\n",
    "        value_loss = F.mse_loss(return_batch, values)\n",
    "\n",
    "        loss = action_loss + self.value_loss_weight * value_loss - self.entropy_loss_weight * dist_entropy\n",
    "\n",
    "        return loss, action_loss, value_loss, dist_entropy\n",
    "\n",
    "    def update(self, rollout):\n",
    "        advantages = rollout.returns[:-1] - rollout.value_preds[:-1]\n",
    "        advantages = (advantages - advantages.mean()) / (\n",
    "            advantages.std() + 1e-5)\n",
    "\n",
    "\n",
    "        value_loss_epoch = 0\n",
    "        action_loss_epoch = 0\n",
    "        dist_entropy_epoch = 0\n",
    "\n",
    "        for e in range(self.ppo_epoch):\n",
    "            data_generator = rollout.feed_forward_generator(\n",
    "                advantages, self.num_mini_batch)\n",
    "\n",
    "            for sample in data_generator:\n",
    "                loss, action_loss, value_loss, dist_entropy = self.compute_loss(sample)\n",
    "\n",
    "                self.optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_norm_max)\n",
    "                self.optimizer.step()\n",
    "\n",
    "                value_loss_epoch += value_loss.item()\n",
    "                action_loss_epoch += action_loss.item()\n",
    "                dist_entropy_epoch += dist_entropy.item()\n",
    "        \n",
    "        value_loss_epoch /= (self.ppo_epoch * self.num_mini_batch)\n",
    "        action_loss_epoch /= (self.ppo_epoch * self.num_mini_batch)\n",
    "        dist_entropy_epoch /= (self.ppo_epoch * self.num_mini_batch)\n",
    "        total_loss = value_loss_epoch + action_loss_epoch + dist_entropy_epoch\n",
    "\n",
    "        self.save_loss(total_loss, action_loss_epoch, value_loss_epoch, dist_entropy_epoch)\n",
    "\n",
    "        return action_loss_epoch, value_loss_epoch, dist_entropy_epoch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updates 100, Num Timesteps 103424, FPS 522,\n",
      "Mean/Median Reward -19.6/-20.0, Min/Max Reward -21.0/-17.0,\n",
      "Entropy 1.66133, Value Loss -0.00956, Policy Loss 0.03579\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNwAAAFoCAYAAACMr+bUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XmYXHWZt/H76U4IyBISA4oiRAYFRdRRRAWRJaCIijMioiMqDIzbqLiMjgtCEMcN1HFBfRUEFXBBBhQGGdkCgzjIJrggKBCQVWKAsGbr5/3jnEqqK1XdXdXVXXXS9+e66qo6+3NqIcmX3xKZiSRJkiRJkqTuGOh1AZIkSZIkSdLaxMBNkiRJkiRJ6iIDN0mSJEmSJKmLDNwkSZIkSZKkLjJwkyRJkiRJkrrIwE2SJEmSJEnqIgM3SZIkqU5EfCoiMiKOb/O4k8vjDp+o2iRJUjUYuEmSpHGLiJPKoKHxsSQifhMRx0TE5r2usxsa7u89I+w3rW6/3bp4/d1avNeNj99065qaWiJiy4h4qO679JJe1yRJUtVM63UBkiRprbIcWFy+DmAT4Dnl49CIeHVmXtqr4ibAxyPihMx8pEfXXwSsHGGbJtedwA1U/73/GrB+r4uQJKnKbOEmSZK66bLMfGL5eAKwAfAW4H5gY+C0iFivpxV21xOAlq3cJsEL6t7vxseePaxrSsrMD2fmtpn5zV7X0qmI2A94FXB5r2uRJKnKDNwkSdKEycxHMvP7wHvLVU8E/qGHJXXTz8vnD0fERj2tROqCiNgQ+DKwBPhQj8uRJKnSDNwkSdJk+DEwVL5+fv2GiHhCRHwhIv4YEY9ExAMR8euI+GBEzGh2srox4+ZHxGBEvC8iri2PXxwRZ0fEDiMVFBE7R8R/l/s/XB7/vogYqD//CKf4f8BfgNnAB8f8TgyvYaPyHq4tx8x6KCKui4ijImJmJ+cc4VqrBvSPiHUj4hPltWpjdW1Q7rdBRBwQEadGxO8j4v6IeDQi/hQR34yIrVucv37Mus0j4hnlOe4q39+rI+JNdftHRLwjIq4qa/hbuf+IY/1FxFMj4msRcWP5eS+JiCsj4sMR8bgWx2wUEUeWNTwYEUsj4s6IuCIiPh8Rz2zzvTy8vM9HI+JVzd7jhv23LtevKJd3iYhzImJReQ/XRMS7IqLtv5uX35+MiP8bZb83l/vdFRGDLXb7NPBk4AjgrnZrkSRJqzmGmyRJmnCZuTQiFgGbAqtag0XEjhQtxWaXqx4E1gFeUD7eHBEvy8y/tjj1NOBsYG+K8eOWArOAVwLzImKPzPxV40ER8RbgRFb/z8f7gWcCXwJeStHCZzRLgaOBbwHvj4ivZObfxnBcrYatgfOBLctVtXHgti8fB0XEnpn5p7Gec4weB1xKEXwuAx5t2H4oxftQ8yDF+7R1+XhTROybmReNcI0XA9+h6FL8ALAe8PfAyRExB/gq8ENg/7KG5RTfgTcCL46I52XmfY0njYj9ge8DtSD20fL188vHP0XEXpl5b90xs4BfAduUq4bKmp4AbAbsUNYwpplFI+ILwAeAh4DR3odmx78eOBUYpPjerQM8FzgO2CMiDsjMVuPyNXMKcCTwwojYKjNvbrHfG8vnHzU7fxlQvwu4lmIMt6e2UYMkSWpgCzdJkjThohi3bZNy8f5y3SzgTIqg5bfAjpm5EUVIsz9wH8VkC6eMcOp/BXYEDgA2yMwNy2N+B6xL0T2usZZtgW9T/D3oHOCpmTmLIgh8L/Bq4DVjvLUTgT8DGwL/PsZjiIh1gNMpwra/AC+juO8NgD2B24AtgDOiRSu/cXgPRZjyeor3bGNgK+Cxcvsi4CsUodnG5WeyHkUg+cOyxlNj5LH4vg1cQPHebkwRgh5fbvsU8Eng5cCbKN67DYFdgXuAuTTpzhgRL2J1UPUZYIvMfBxFgLgTcCXFZ39Sw6Hvpwjb/grsA8zIzNkU349tgI8Ct4xwL7XrD0TE8RRh22JgXrthG8V37njgXGBu+b2bCXwESGA/2mwtWQayV5WLb2y2T0Q8HtirXDy1yfZBiuA4gHe2GfhJkqQmDNwkSdJkOITiH/OwejD2d1O0MLofeFlmXgGQmSsz8yfAG8r99oyIPVqcd2PgNZn548xcVh5/HXBQuf0FEbFlwzEfpWhV9DvgHzNzYXnco5n5VeDj5XlHlZkrgKNq9xMRm43lOIqA8NnACmCfzDwvV7uAIhhaDmxHEUq1ckVE3N3i0WpcuQ2A12fmaZm5vLyPW8p7ITNPzszDMvP/MvOBct1QZl4P/BOwgGIsvteOUNddwH517+0DwNspgq0NKN7jd2bmqZm5rLzvSyg+G4DXNTnnlyhaNH4gMz+WmX8pz72ybMW4N3A3sE9EPLfuuBeVz5/PzJ/X3efyzLwxMz+bmSeMcC9ExHSKsPGQ8hq7ZeavRzqm1ako3oPXZuatZR0PZ+bnKLpzAnx0lDCzmVqI9k8ttu9P8d7d1KLuwyhaIH6nWYtQSZLUPgM3SZI0IcoxuuZGxL8Bny9X3wqcVb6uhSrHZ+bdjcdn5i8ougJC0Rqrmf/NzEubHHsVcHu5uF1dTQOsnrThP2shXYOvAQ+3uF4zpwK/p2gF9vExHlO79zMz83eNGzPz98BPysVW9w4wh6JrZLNHq7/nXV2Gem3LzAT+u1zceYRdj2lsJZWZQ0CtRditwA+aHFera+uIWLe2MiK2oQjOHqZoidWstr8B/1Mu7lW3qdY9eKxh6DDluHA/owitbgV2yczfdnKu0rEtvnfHUnRt3RiY1+Y5f0jRVfaZEfHsJttrLd+atW57CkVovJg2WmlKkqSRGbhJkqRu2rU2cD5FAHALcAxFGHUX8A+ZuazsUvms8piRuuVdWD4/r8X2K0Y49o7yeVbduq1YPYbcGkEdFDOrsrqL3qjKIOnIcvFfmrSoa6Z2P+O5dyi6bEaLx/0tjhm1BVNEbFFOJnBVFJMmrKz7XI8pd3vSCKdoFUjVxuL7fRneNbqnVgJFV8uancrnGcCtrVr1sTrIfErdseeUz++PiO9GxN5RzMY5FjMpQry9gRuAl2Tmn8d4bCsLmq0sP69ry8WRPvNmx94JXFwuDmvlVk5CsUu5uEbgRjGe3gbAR9oZg1CSJI3MwE2SJHXTcorQ5B6Krnc3AecBHwa2y8zflPvNZvXfQ+5oPEmdWiu1TVpsf3CEY2tjkk2vWzen7vVIszDeOcK2Zv4LuJqiq+oRY9i/dj9juffHR0SMsF+77h1pY9l99w8U46g9jyKgfJDVn2utxdj6I5ym1Xu7cozbYfjnVmudNo3WLfqeUFdT/WylJwInUHzf3kIxScf95Yyl8yPiiSPcx+uAl1C0PNs7M28fYd+xGum7Vfs+rPq+l3U2Cxjf13BsLUx7Q8P35Q0UAeY1mfnH+gMi4h8pxiu8nNVj7EmSpC4wcJMkSd10WWY+sXxslplbZ+bLMvOYZrNOlro9KcBIuhlcrVK21vpEufjWiHj6GA+dzHuvaTkgfjlBw/cpgqtfUIRN62XmxrXPlSI8hQl6L1uo/Z31ihFa9NU/Dq0dWI4PdyjFmHlHU7QEW0YxZtmRwJ9GGCNwAUVwvA5wfH031wnS7D3dlObh4gYN+/2E4r62ZHWLQGjRnbQM5b5CMVnDh4D1I2KD2oPhoeV65fqJvn9JktYaBm6SJKkXFlN0OYUiIGhl8/J5xFZZbag/z0hjerU93ldmngP8kmIWzaNG2b1Wx1ju/W8tul9OhJ0puoouopiM4peZubRhnydMUi31al1Ntyln1GxbZv42M4/IzN0oxknbl2LsvQ2A77U4700U46ndWz7/V9kdejzG8r1b9T3NzM1bhIqfqj+w7JL683LxjQBl8Ps8ilDthw3XGqT4jgVwCUUrxvrHtXX7/qJcd/aY71KSpCnOwE2SJE26ctD42mQBu4+wa63l0dVduvTNrO4S+ZJmO5QzRD6/w/MfXj4fAGw/wn61+5nMex+LWsj3x8x8rMU+e05WMXVq485tRPsTCqwhM5dm5lkUnxPAk4G/a7HvHyjueTHwCuC0ctbSTu3abGVEzASeUy52+pnXWrG9PiKmsbp12yVd6g4rSZLGyMBNkiT1Sm0WzoMiYo1WPxHxMuDF5eKPu3HBcoKDn5aLh7UITt7Fmt31xnr+BRQzbQZF98VWavf+ioj4+8aNEbEdqycA6Mq9j9ED5fM2zVpyRcQ+rB6Af9KUM7leWS5+vpw5tKmIeFx97aO0SHu07nXL7r2ZeR3FzKf3U7SMO7XTlnbAv7X43n2grOF+4PwOz30WRUu0TSiCyZazk2bmipG65QJPq9t9l3J9L8JWSZIqycBNkiT1ytcoBs9fDzg3InYAiIjBiNiP1V3gzs/MC1ucoxOfoRjranvg9NqsohGxbkT8a7m91QyfY/Hx8vmVI+zzI+C68vWZEbFnbaD7iJhHMbPmdIouj6eMo5Z2XUoRQm0CfLc2oUBErBcR/0IR/vVqJst3U3xuzwEuiYg9aqFXRAxExLMi4giKbqCb1h13UUR8OSJ2qR+DLCKeBXynXLydYqKIljLzauDlFC0kXwd8PyLa/bt0UrSkOz0itijrWD8iPsTq1pGfGaF14cgnz3wUOLNc/DSwDcVEJj9peZAkSZoQBm6SJKknykkU/gG4j2JA+ysiYgnwEEVAMIsilHpTl697PfAOivDj1cDCiFhMEaR8jSKw+Fm5e+P4ZWM5/+WMMtZV2aV2P+BWYAuKmVwfioiHKVo3bQHcBry2yRhqEyYz/8bq4OcNwF0RcT/Fe/Mt4AbgUy0On+jaLqd4zx6k6PJ7AfBIRCyimJH2txRj5z2R4rOtmQm8l2KcsocjYnFE1PbfFXgYeHNmtpxMoq6GXwP7UHxH3wic0OYMskPAv5TnuLX83t0PfJ7i7+WnA19o43zN1FqzPa98PjczF4/znJIkqU0GbpIkqWfKAOOZwJeAGylada2g6D74IeCFmfnXCbjuicBLgXMpulHOoGjhdBhF0DSz3LXTlm6fYHjo06yGP1O01vokq8ezo3x9NPDszLyxw+t3LDO/COxPMW7ao8A04I8U97QzRdjUE5l5NkVXx08D11AEbRtTBIK/LGvcNjPvqDvsYGA+xYyjt1G0qBwCrqeYpfNZZVfgsdbwS+BVwCPAQcA32wndMvNHFOPz/ZziO7IC+A3wr8DrxxL8jeI8oP43s0Z3UkmSNPFi8ia9kiRJ6n9leHIr8BRg93bCGKmZiNga+BOwMjOn9boeSZI08WzhJkmSNNwbKMK2JcCve1yLJEmSKsj/wyZJkqaciPgYxVhgZwJ3ZOZQRMwC3kIxaQLA1zPzkV7VKEmSpOqyS6kkSZpyIuJkVk/GsIxi4PyNgdpYXOcDr+50tkipnl1KJUmaevwDX5IkTUVfp+gy+hJgM4qwbTHFrKgnA9/LzBW9K0+SJElVZgs3SZIkSZIkqYts4dZH5syZk3Pnzu11GZIkSZIkSWuNq666alFmbjKZ1zRw6yNz587lyiuv7HUZkiRJkiRJa42IuHWyrzkw2ReUJEmSJEmS1mYGbpIkSZIkSVIXGbhJkiRJkiRJXWTgJkmSJEmSJHWRgZskSZIkSZLURQZukiRJkiRJUhcZuEmSJEmSJEldZOAmSZIkSZIkdZGBmyRJkiRJktRFBm4Vc+M9D3LOb+/qdRmSJEmSJElqYVqvC9DYLHpoKS/69AUMRLBs5RALP/vKXpckSZIkSZKkJgzcKuLymxezYiiBZLsnbdTrciRJkiRJktSCXUorImL168dvMKN3hUiSJEmSJGlEBm4VES1eS5IkSZIkqb9UPnCLiOkRcVhEnBgRv4mIZRGREXHoKMdtFREnRMRfymPujogfRMS2bV7/oPJ6rR7vGN8dSpIkSZIkqUrWhjHc1gf+s3x9D3A38JSRDoiI5wEXARsBFwI/LI/ZD3h1ROyZmf/XZh0/BX7TZP2VbZ6nqfoupWETN0mSJEmSpL61NgRujwD7AL/JzLsiYj5w5CjHnEARtn0gM79UWxkRLwYuAb4XEdtl5vI26jgzM09qq/K2RJNXkiRJkiRJ6jeV71Kamcsy8+eZeddY9o+IrYDnAn8Fvtxwrl9RtFR7GrB3t2sdj+Et3IzcJEmSJEmS+tXa0MKtXU8snxdm5lCT7TeXz/OAs9o473Mj4n3AusAdwEWZeXvnZbZm3CZJkiRJktS/pmLgtqh83jIiIjOzYftW5XNbkycAhzUsr4yI44H3ZeZj7RbZyJBNkiRJkiSpGirfpbRdmXkjcCPwBOA99dsi4oXAa8rFWWM85S3lebahmMDhScDrgYXA24HvjHRwRLwtIq6MiCvvvffekfarez3GyiRJkiRJkjTp+iJwi4iFEZFtPE4e5yXfDiwFvhwR50XEMRHxA4oJE/5Q7rNyLCfKzIsz82uZeWNmPpKZd2XmacDuwH3AGyPiOSMc/63M3CEzd9hkk01aXidGWJIkSZIkSVL/6JcupTcB7XS7vHM8F8vMBRGxI3A4sGv5+AvwKeBaiokT/jrOa/wlIs4B3gS8tDxvx4ZPmjCeM0mSJEmSJGki9UXglpnzenDN6yi6fg4TEUeVL6/owmVqfUTXH++Jpg2uboxo3iZJkiRJktS/+qJLab+IiBnAW4Ah4IddOOULy+ebR9xrDKYPGLNJkiRJkiRVwZQM3CJi/YgYbFg3HfgGMBf4Rmbe1LB9s4jYNiJmNqzfpcn5IyI+CryYYlbUc8db8+CAkyZIkiRJkiRVQV90KR2viPgIsG25+Nzy+eCIeEn5+tLMPL7ukN2B4yPifIqx2zYC9qEI2/4b+Lcml/kM8FbgYOCkuvWXRMSNFF1Q7wBmAjsDzwIeAd6UmUvGc38A0wbrAjc7lUqSJEmSJPWttSJwA/ammPig3k7lo6Y+cLsR+GV5zKbAoxSTGhwFfC8zh9q49rHAjsAewGyK7qi3AccBX8zMcXcnBYiwhZskSZIkSVIVrBWBW2bu1ub+NwL7tXnMQcBBTdZ/qJ3zdIOBmyRJkiRJUv+akmO4VdHpV93e6xIkSZIkSZI0BgZuFXHK5beteu0YbpIkSZIkSf3LwK2KzNskSZIkSZL6loFbBZm3SZIkSZIk9S8DtwoKZ02QJEmSJEnqWwZukiRJkiRJUhcZuFWQ7dskSZIkSZL6l4FbBdmjVJIkSZIkqX8ZuFWQeZskSZIkSVL/MnCrICdNkCRJkiRJ6l8GbpIkSZIkSVIXGbhVkO3bJEmSJEmS+peBWxWZuEmSJEmSJPUtA7cKChM3SZIkSZKkvmXgVkHOmSBJkiRJktS/DNwkSZIkSZKkLjJwqyAbuEmSJEmSJPUvA7cKskupJEmSJElS/zJwqyAnTZAkSZIkSepfBm4Vse9zntTrEiRJkiRJkjQGBm4VsdnMdVe9tkupJEmSJElS/zJwq4ihzFWvDdwkSZIkSZL6l4FbRdTlbThPqSRJkiRJUv8ycKuIobrAzRZukiRJkiRJ/cvArSKGhjdxkyRJkiRJUp8ycKsgG7hJkiRJkiT1LwO3inDSBEmSJEmSpGowcKuIYYGbbdwkSZIkSZL6loFbRaSTJkiSJEmSJFWCgVtFDDlngiRJkiRJUiUYuFVEDutSKkmSJEmSpH5l4FYRw7uUGrlJkiRJkiT1KwO3iqifNEGSJEmSJEn9y8CtIoacNEGSJEmSJKkSDNwqIrGFmyRJkiRJUhUYuFXEsDHcnDZBkiRJkiSpbxm4VcSwWUrN2yRJkiRJkvqWgVtFDBvDrXdlSJIkSZIkaRQGbhUxZAs3SZIkSZKkSjBwqwinTJAkSZIkSaoGA7eKGD6Gm03cJEmSJEmS+pWBW0UMDa1+bdwmSZIkSZLUvwzcKiJx1gRJkiRJkqQqMHCriOGzlJq4SZIkSZIk9SsDt4qoH8NNkiRJkiRJ/cvArSLq8zbnTJAkSZIkSepfBm4VMVQ/S2kP65AkSZIkSdLIDNwqYsgWbpIkSZIkSZVg4FYR9SO4OWmCJEmSJElS/zJwqwgnTZAkSZIkSaoGA7eKcNIESZIkSZKkajBwqwgnTZAkSZIkSaoGA7eKGLJLqSRJkiRJUiUYuFXEsLzNPqWSJEmSJEl9y8CtImzgJkmSJEmSVA0GbhXhGG6SJEmSJEnVYOBWActWDHHlrfetWrZHqSRJkiRJUv8ycKuAH11x27DlR5et7FElkiRJkiRJGo2BWwXcsuiRYcvnX39PjyqRJEmSJEnSaAzcKmBwoHHZPqWSJEmSJEn9ysCtAqJh0LbBAT82SZIkSZKkfmVyUwGNkyRMs4WbJEmSJElS3zJwq4CBhsRtwMBNkiRJkiSpbxm4VUBjvmYLN0mSJEmSpP5l4FYBjS3cnDRBkiRJkiSpf03r1okiYlvgFcAjwA8z84FunXuqWyNwaxzUTZIkSZIkSX2j7RZuEXFERNwVEbPr1u0JXAMcC3wduDoiHt+9Mqe2xsBt2qCBmyRJkiRJUr/qpEvpK4A/ZubiunWfARI4EvgG8FTgsPGXJ1hzDLfGAE6SJEmSJEn9o5PAbS5wfW0hIp4MPB/4emZ+KjPfDVwI/ENXKhSN+Zp5myRJkiRJUv/qJHCbBdS3btuZonXb2XXrrgK2GEddqhMNCZst3CRJkiRJkvpXJ4HbvcCT65Z3B5YDl9etW6fDc7clIp4WEf8eERdGxF8iYllE3BMRP42I3Uc59q0R8euIeCgiHoiIBRHxqg7reFV5/APl+S6PiLd2dldragzYjNskSZIkSZL6Vyeh2G+AfSPiWRGxNXAAcGlmPlq3z1zgri7UN5qjgc8CTwDOAb4A/BJ4JXBhRLy32UERcSxwErAZ8G3gZGB74KyIeHc7BZT7nwU8qzzPt4EnASeV1xm3xjHcbOAmSZIkSZLUv6Z1cMzngYuAa+vWfaH2IiLWBXajCMAm2rnA5zLzmvqVEbErcB5wTESclpl31W3bCfggcBPwgsy8r1x/DEVX2GMj4uzMXDjaxSNiLsXMrIuBHWrHRMQngSuAD0bE6Zn5q/Hc5GBj4mYbN0mSJEmSpL7Vdgu3zPxf4FXAmcAZwOsy8+d1u+wELCy3TajMPKkxbCvXXwwsoOjaulPD5neUz/9RC9vKYxYCxwEzgIPHWMI/l/t/rT6gK8/76YbrdaxxDDdbuEmSJEmSJPWvTlq4kZnnUrQua7btQuDvx1NUlywvn1c0rN+jfG5W/8+BT5T7HDmGa4x2rvp9OrZGl9LxnlCSJEmSJEkTZsInNuiFiNgSmAc8AlxSt359igkfHqrvZlrnT+Xz08d4qW3K5xsbN5TnfxjYPCIeN8bzNdU4acLs9dcZz+kkSZIkSZI0gUZt4RYRL+305Jl5yeh7dVdEzABOoejq+eH6bqPAzPL5gRaH19ZvPMbLjeV865f7PdKi3rcBbwPYYostmp6ksYXbR16x7RjLkyRJkiRJ0mQbS5fSBUB2eP7B0XaIiIXAlm2c85TMPLDFuQaB7wM7Az+imNCgE53e7xoljXa+zPwW8C2AHXbYoel+tZWXf2we604bZObjpnepPEmSJEmSJHXbWAK3T7JmYPRCYG+KmT4vBe4Gngi8BPg7ivHLfj3GGm4CHhvjvgB3NltZhm0nA/sDPwYOzMzGumst0WbS3Ggt1ho9AMwpj/tbk+0blc9Lxni+pmp3MW0gDNskSZIkSZL63KiBW2bOr1+OiBcBHwUOA47LzKG6bQPAe4DPUgR1o8rMeW3U21RETANOpQjbTgXekpkrm1zr4Yi4A3hyRGzWZBy3p5XPa4zJ1sINFIHb04FfNdS0GUV30tszs2l30rGq5YaNs5VKkiRJkiSp/3QyacLRwPmZ+dX6sA0gM4cy88vABYwxcBuviFgH+AlF2PY94M3NwrY6F5bPezfZ9oqGfUbTzXO1VGumZ9wmSZIkSZLU/zoJ3HYEfjPKPtcCL+rg3G0pJ0g4A3gNcAJwcGMI2MQ3y+ePR8SsunPNBf4VWAqc2HCdORGxbUTMaTjXieX+7y6Pr+0/C/hYw/U6VutSagM3SZIkSZKk/jeWMdwaBcU4bSPZuoPzduKbwD7AIuAO4Igm3S4XZOaC2kJmXhYRXwQ+AFwXET8B1gEOAGYD78nMhQ3neDdwJHAUML/uXLdExIeArwBXRsSPgGXA64DNgS9k5q8Yp9Ut3EzcJEmSJEmS+l0ngdtlwH4R8arMPLtxY0TsC7wWOG+8xY3BU8vnOcARI+y3oH4hMz8YEddRBGlvA4aAq4Fjmt3TSDLzq+VMq/8GvIWi1eAfgMMz87vtnGuEaxQvzNskSZIkSZL6XieB28eBS4CfRsTF5et7gCcAuwIvBR4t95tQmbnbOI79LjCmQKycOGL+CNvPAs7qtJaxGjBwkyRJkiRJ6nttB26ZeVVE7AV8B9itfCSr21/dABySmdd0qcYpb8hZSiVJkiRJkiqjkxZuZOZlwLYRsRPwPGAm8ABwdblNXWSPUkmSJEmSpOpoO3CLiJcCSzLzN2W4ZsA2wVZNmmDiJkmSJEmS1PcGOjjmIoqJBjRJVrdwM3GTJEmSJEnqd50EbosoJkXQJElqY7j1uBBJkiRJkiSNqpPAbQGwU5fr0AhqLdwkSZIkSZLU/zoJ3A4HtomIoyNiercLUmu2cJMkSZIkSep/ncxS+lHgd8DHgEMi4lrgblaP7V+TmXnIOOsTkGUTtwETN0mSJEmSpL7XSeB2UN3rJ5aPZhIwcOuCoVWTJkiSJEmSJKnfdRK4PbXrVQiAh5au4LHlK1l3+uCw9atmKbWFmyRJkiRJUt9rO3DLzFsnohDBLYse5sRfLuSdu/3dsPWrZintRVGSJEmSJElqSyeTJmgC/fHuJWusW93CbZKLkSRJkiRJUts66VK6SkQMAnOAGc22Z+Zt4zn/VDQ4sGaqVpuNwi6lkiRJkiRJ/a+jwC0itgc+C+xOi7CNIicaV6A3FU0faNLoMBsngJUkSZIkSVK/ajsQi4htgcvKxfOAVwPXAvcAz6No8XYRYOu2DkwbbN7CzcZtkiRJkiRJ1dDJGG6fAKYDO2Xma8p1Z2Tm3hQzmJ4IPBM4ojslTi3TmnUpTSdMkCRJkiRJqopOArfdgLMz87d16wIgMx8G3g7cBxwSsIxvAAAgAElEQVQ97uqmoMEmXUqTZMAmbpIkSZIkSZXQSeA2B/hT3fIK4HG1hcxcQdGl9GXjK21qmt6sS6lDuEmSJEmSJFVGJ4HbYmCDuuVFwBYN+ywDZnZa1FQ20KRLKTiGmyRJkiRJUlV0ErjdBMytW74K2CsiNgWIiPWB1wC3jLu6KahZ3mYDN0mSJEmSpOroJHD7BbB7GawBfBOYDVwTEacBvwW2BI7vTolTS6ux2sJpEyRJkiRJkiqhk8Dt28AhwHoAmfnfwPvK5f2ATYHPAV/pUo1TSrNYzTHcJEmSJEmSqmNauwdk5l3AjxrWfSUijqOYUOGvmUZEHWvSwi3J5kmcJEmSJEmS+k7bgVsrmbkSuKdb55uqWuVq5m2SJEmSJEnV0HaX0og4KSIOjIgnT0RBU13TMdxsLyhJkiRJklQZnbRwewvwZoCI+BNwIXABcFFmLu5ibVNSizkTWq6XJEmSJElSf+kkcNsO2APYE9gVeAfwdiAj4jpWB3CXZObD3Sp0qmg6acKkVyFJkiRJkqROdTJpwvXA9cBxERHA84F55WMn4DnA+4HlwLrdK3VqGBhoMmlCJuEobpIkSZIkSZUwrkkTytlIrwSujIifA68EDgM2BaaPv7ypp7Hr6F0PPMrv7ljCyiHbuUmSJEmSJFVBx4FbRDyV1S3b9gDmUPSIXAicQNGtVOOQmbz4Mxf2ugxJkiRJkiS1oe3ALSK+TRGybUkRsN0DnE85dltmLuxmgVNNfdfR+kZtT954vR5UI0mSJEmSpHZ10sLtEIpx/M8DjszMy7tbkmqKHruFx60z2MNKJEmSJEmSNFYDHRxzKcWECC8DLomI/42I+RGxS0Q4bts41Y/hVj9q22CTyRQkSZIkSZLUf9oO3DLzpcAsYG/gyxQzkR4OXAzcFxHnRsSHIuL5Xa10iqiP1eoauBm4SZIkSZIkVURHkyZk5qPAL8oHEbExsDvF5An7A3tRNNAa1yyoU13WtXEzcJMkSZIkSaqGcQdiETGLImzbk2IyhU3He04VbOEmSZIkSZJUPZ3MUroesAtFuDYPeC5FT8gAlgBnAxeUD3XJNAM3SZIkSZKkSuikhdt9wHSKgO0xYAFFuHYhcEVmDnWtuiloqK5VW30Lt4EwcJMkSZIkSaqCTgK3a4DzKQK2yzJzaXdLmtrqx22rfz1t0MBNkiRJkiSpCtoO3DLzxRNRiAppCzdJkiRJkqRKGxjvCSJiVkQ8pRvFCDLrW7it5hhukiRJkiRJ1dBR4BYRG0TEFyLibmARcEvdthdGxDkR8bxuFTmVDG/htnphcGDc2agkSZIkSZImQdspTkTMBH4FvB+4E7ieYgKFmt9SzGL6xm4UONUMmzShbv2geZskSZIkSVIldBLjfBzYDjgoM58HnFa/MTMfAS4G5o2/vKln2KQJdYnboF1KJUmSJEmSKqGTwO21wP9k5vdG2OdW4MmdlTS11Yds9U3cwkkTJEmSJEmSKqGTwG1z4LpR9nkImNnBuae84ZMmrH7tLKWSJEmSJEnV0Eng9iCw6Sj7PJViMgW1aVgDt/oWbpNeiSRJkiRJkjrRSeB2BfCqiNiw2caI2AzYB7h0PIVNVdli0gSHcJMkSZIkSaqGTgK3LwOPB86JiGfUbyiXTwPWBb4y/vKmnqH6LqVpl1JJkiRJkqSqmdbuAZn5PxExH5gP/A5YDhARi4BZFL0f/z0zL+temVNHizkTnDRBkiRJkiSpIjpp4UZmfhKYB/wMuA9YSZEPnQPsmZnHdK3CKWZYl9K613YplSRJkiRJqoa2W7jVZOZFwEVdrEU4S6kkSZIkSVLVddTCbSwiYpOJOvfaLFssDEzYJyVJkiRJkqRu6nqMExEzI+LTwE3dPvdUMLyF22qO4SZJkiRJklQNbXUpjYgtgedTTJTw68y8p27busD7gX+jmDzhkS7WOWUMOYabJEmSJElSpY25hVtEfIWi1dppwJnAwoh4V7ltN+AG4FPAesCXga26XexUMGzSBMdwkyRJkiRJqpwxtXCLiLcC7waGgOuBALYBvhIRDwP/Dxgsnz+VmXdOTLlrv/qQrT58M26TJEmSJEmqhrF2KT0IWAbsnpm/AoiIlwLnAScAtwOvzszfTkSRU8nwFm6rOYabJEmSJElSNYy1S+mzgTNqYRtAZl5C0bU0gH82bOuOYZMmpF1KJUmSJEmSqmasgdtM4M9N1v+pfP5Vk23qQH2rNidNkCRJkiRJqp6xBm4DFDOTNloOkJmPdq2iKW6oPmWrM2DiJkmSJEmSVAljnqWU4Y2vNEGGjeFWP2mCeZskSZIkSVIljHXSBID5ETG/2YaIWNlkdWZmO+cXDV1KcQw3SZIkSZKkqmknEGs38TEh6sDwSRNWr7dHqSRJkiRJUjWMKXDLzHa6nmochnUprVtvCzdJkiRJkqRqMEjrM8PHcFu9EAZukiRJkiRJlWDg1mf+9vAyAB5bvpIzrrlj1XrjNkmSJEmSpGpwUoM+s2zlEAAv+I/zefCxFavWD6WTxEqSJEmSJFWBLdz6zOOmDwIMC9sALr7x3l6UI0mSJEmSpDYZuPWZlS1asjmGmyRJkiRJUjUYuPWZoaEWgdsk1yFJkiRJkqTOVDpwi4inRcS/R8SFEfGXiFgWEfdExE8jYvdRjn1rRPw6Ih6KiAciYkFEvKrN6+8WETnC47Pt3lOrFm4DJm6SJEmSJEmVUPVJE44GDgD+AJwDLAa2AfYF9o2IwzLzK40HRcSxwAeB24FvA+sAbwDOioj3ZObX2qzjYmBBk/WXtnkeVrZo4TZgl1JJkiRJkqRKqHrgdi7wucy8pn5lROwKnAccExGnZeZdddt2ogjbbgJekJn3leuPAa4Cjo2IszNzYRt1LMjM+eO6k1Kr2UgN3CRJkiRJkqqh0l1KM/OkxrCtXF9rcbYOsFPD5neUz/9RC9vKYxYCxwEzgIMnot6xaNXCTZIkSZIkSdVQ6cBtFMvL5xUN6/con89tcszPG/YZq60j4t0R8bGI+OeIeFqbx68yNNTpkZIkSZIkSeoHVe9S2lREbAnMAx4BLqlbvz7wZOCh+m6mdf5UPj+9zUu+qXzU13A68C/1rejGotWkCZIkSZIkSaqGta6FW0TMAE6h6Bo6vyHwmlk+P9Di8Nr6jcd4uXuBjwDbAxsCmwCvAK4B9qOYhGHE9zgi3hYRV0bElWCXUkmSJEmSpKrreeAWEQsjItt4nDzCuQaB7wM7Az8Cju2wrDGlXpn5+8z8XGb+LjMfysxFmXkusBtwS1nHq0c5x7cyc4fM3AFaT5qQYytJkiRJkiRJPdYPXUpvAh5rY/87m60sw7aTgf2BHwMHZq6RXtVasM2kudFawI1JZi6JiFOBjwMvBX461mNbtXCz5ZskSZIkSVI19Dxwy8x54z1HREwDTqUI204F3pKZK5tc6+GIuAN4ckRs1mQct9pkBzeOtyaK7qYA67dzUKtgzbxNkiRJkiSpGnrepXS8ImId4CcUYdv3gDc3C9vqXFg+791k2ysa9hmPF5XPN7dzUOsupZIkSZIkSaqCSgdu5QQJZwCvAU4ADs7MoVEO+2b5/PGImFV3rrnAvwJLgRMbrjMnIraNiDkN63duNilCRBwIHAAso+jeOmatWrit2TtWkiRJkiRJ/ajnXUrH6ZvAPsAi4A7giIho3GdBZi6oLWTmZRHxReADwHUR8RNgHYqAbDbwnsxc2HCOdwNHAkcB8+vWnwIMRMRlwO3AusALgB2BFcDbm5xrRK26jjqGmyRJkiRJUjVUPXB7avk8BzhihP0W1C9k5gcj4jqKIO1twBBwNXBMZp7dxvW/AexJMRvpHCAogr+TgP/MzGvbOBfgGG6SJEmSJElVV+nALTN3G8ex3wW+O8Z95zO8ZVtt/eeAz3VaQzO3LX6k6fqBNRruSZIkSZIkqR9Vegy3tdUd9z+6xroDX7RlDyqRJEmSJElSuyrdwm1t9cjSFcOWz//ArvzdJuv3qBpJkiRJkiS1wxZufaTWa3Rlw4ykGz9uOk0mg5AkSZIkSVIfMnDrI7VQbcXK4YHboGGbJEmSJElSZRi49ZFarjbU0MJtwBkTJEmSJEmSKsPArY+s6lI61NDCzcBNkiRJkiSpMgzc+kiUkVtjCze7lEqSJEmSJFWHgVs/KXO1lUPDVw/4KUmSJEmSJFWGUU4fiVWBmy3cJEmSJEmSqsrArY/UYrU1upQ6hpskSZIkSVJlGLj1kdoYbo0t3MIWbpIkSZIkSZVh4NZHVnUpbWjhJkmSJEmSpOowcOsjtcBtaMjATZIkSZIkqaoM3PpIrUvpCgM3SZIkSZKkyjJw6yO2cJMkSZIkSao+A7c+5BhukiRJkiRJ1WXg1kdWTZpgCzdJkiRJkqTKMnDrI7Ux3IZs4SZJkiRJklRZBm59pNbC7asX/rm3hUiSJEmSJKljBm59ZHCgSNxuvvfhHlciSZIkSZKkTk3rdQFabVoZuNW8+UVb8pFXbNujaiRJkiRJktQJW7j1sZnrTWf9GWaikiRJkiRJVWLg1sciRt9HkiRJkiRJ/cXArY+Zt0mSJEmSJFWPgVsficYmbTZxkyRJkiRJqhwHCOtjA+ZtkiRJkiRNqKVLl7J48WIefPBBVq5c2etyNIrBwUE23HBDZs+ezYwZM3pdTksGbn0s7FQqSZIkSdKEWbp0KbfddhuzZs1i7ty5TJ8+fc3eZ+obmcny5ctZsmQJt912G1tssUXfhm52Ke0jjT9pf+OSJEmSJE2cxYsXM2vWLObMmcM666xj2NbnIoJ11lmHOXPmMGvWLBYvXtzrkloycOtj/swlSZIkSZo4Dz74IBtttFGvy1AHNtpoIx588MFel9GSgVs/cc4ESZIkSZImzcqVK5k+fXqvy1AHpk+f3tdj7hm49ZE1u5SauEmSJEmSNJH8t3c19fvnZuDWR2avv86w5T7/7kiSJEmSJKkJA7c+MtCQsDlLqSRJkiRJUvUYuPUxW7hJkiRJkiRVj4FbHxswcJMkSZIkSRMsIoY9BgcHmTNnDnvssQennHLKuPevufLKKzn44IPZaqutWG+99dhoo43Yfvvt+dCHPsQdd9wxkbc46ab1ugC1ZpdSSZIkSZI0WY488kgAli9fzg033MCZZ57JRRddxFVXXcUXv/jFjvfPTD7ykY/w+c9/nmnTprHXXnux//77s2zZMi677DKOPfZYvv71r/Pd736X173udZNzsxMsMrPXNai0ww475KI9j1q1fPgrn8Ghu2zVw4okSZIkSVp7XX/99TzjGc/odRk9V5vxszEjuuCCC9hrr70AuPnmm5k7d25H+3/yk5/kyCOPZO7cuZx99tlst912w447/fTTOfDAA1m+fDnnnXceu++++5jqHuvnFxFXZeYOYzppl9ilVJIkSZIkSWuYN28e2267LZnJFVdc0dH+Cxcu5Oijj2b69On87Gc/WyNsA9hvv/340pe+xMqVK3nnO9/J0NBQ1+9lshm49ZkDX7TFqtdLHlvRw0okSZIkSdJUV2vFFmOc2bFx/xNPPJEVK1bwj//4j2y//fYtjzv00EN50pOexA033MDFF188zqp7z8Ctz3xy32eter3VnPV7WIkkSZIkSZrKzj//fG644QYighe84AUd7X/ppZcCsOeee4547LRp09htt90A+OUvfzm+wvuAkyb0mYG6qUkHnaZUkiRJkqSeOOqs3/OHO5f0uowRPfNJG3Hkq9fsotmp+fPnA8MnQchM3v/+97Plllt2tP9dd90FwFOe8pRRr1/b58477+zC3fSWgVsfM3CTJEmSJEmT5aijiokcI4KNN96YXXbZhUMOOYQDDzyw4/3b6ZLabvfVfmbg1sfM2yRJkiRJ6o1uthyrisZZR7ux/2abbcYf//hHbrvttlH3vf3221cdU3WO4SZJkiRJkqQJ8ZKXvAQoxncbycqVK1mwYAEAO++880SXNeEM3PqaTdwkSZIkSVJ1HXTQQQwODnLGGWfw+9//vuV+3/nOd7jzzjvZZptt2HXXXSexwolh4CZJkiRJkqQJsdVWW/Gxj32M5cuXs++++/KHP/xhjX3OPPNMDjvsMAYHB/n617/OwED14yrHcOtja8EYgZIkSZIkaYqbP38+Dz/8MF/84hd5znOew8tf/nK22247li9fzmWXXcbll1/Oeuutxw9+8AP22GOPXpfbFQZukiRJkiRJmjADAwN84Qtf4IADDuC4447jkksu4YILLmBwcJC5c+fywQ9+kPe9731svvnmvS61awzc+pgN3CRJkiRJ0kSbiNlJm9lxxx3ZcccdOzq2aqrfKXYt1tnXV5IkSZIkSb1k4CZJkiRJkiR1kYFbH7NLqSRJkiRJUvUYuEmSJEmSJEldZODWxyJs4yZJkiRJklQ1Bm6SJEmSJElSFxm49THbt0mSJEmSJFWPgVsfy14XIEmSJEnSWi7Tf31XUb9/bgZufej1O2wOwFPnrN/jSiRJkiRJWnsNDg6yfPnyXpehDixfvpzBwcFel9GSgVsf+tx+z+bqT+zF1ptu0OtSJEmSJElaa2244YYsWbKk12WoA0uWLGHDDTfsdRktGbj1oYhg9vrr9LoMSZIkSZLWarNnz+a+++5j0aJFLFu2rO+7KU51mcmyZctYtGgR9913H7Nnz+51SS1N63UBkiRJkiRJvTBjxgy22GILFi9ezMKFC1m5cmWvS9IoBgcH2XDDDdliiy2YMWNGr8tpycBNkiRJkiRNWTNmzGCzzTZjs80263UpWovYpVSSJEmSJEnqIgM3SZIkSZIkqYsM3CRJkiRJkqQuMnCTJEmSJEmSusjATZIkSZIkSeoiAzdJkiRJkiSpiwzcJEmSJEmSpC4ycJMkSZIkSZK6KDKz1zWoFBH3Arf2ug61bQ6wqNdFqCN+dtLk8jcnTS5/c9Lk8jcnTa52fnNbZuYmE1lMIwM3aZwi4srM3KHXdah9fnbS5PI3J00uf3PS5PI3J02ufv/N2aVUkiRJkiRJ6iIDN0mSJEmSJKmLDNyk8ftWrwtQx/zspMnlb06aXP7mpMnlb06aXH39m3MMN0mSJEmSJKmLbOEmSZIkSZIkdZGBmyRJkiRJktRFBm5aK0TE6yLiqxHxvxGxJCIyIk7udV2NIuKZEfHjiPhrRDwWETdExFERsd4Ix0REvDUiFkTE4oh4NCJuKc/z9Mmsv5si4vERcWhEnBERfy7v64GIuDQiDomIvvnvk5+b1mYR8ebyv5kZEYf2up6aiNgpIs4pfz+PRMR1EfG+iBgc4ZjpEfHeiLi8/O/JwxFxY0R8LyI2mcz6pXoRsUtEnB4Rd0XE0vL5FxGxT69rA/+c09olIl5Z/r5uL7+XN0fEaRHx4l7XVuNvTmuLtfHf4RFxUt3fjVs9LhjLdad1/1aknjgceA7wEHA7sG1vy1lTRLwQuBCYDvwE+AuwB3AEMC8i5mXm0oZj1gVOA14F3ACcCjwIPAnYBXg6cONk3UOX7Q98A7gLuAi4DXgC8FrgeOAVEbF/9nigST83rc0i4inAVyn+27lBj8tZJSJeA5wOPAb8CFgMvBr4ErAzxX8/Go+ZDfwc2BG4GvgOsAx4CrAnxX9f7p2E8qVhIuJw4GhgEXA2xZ97c4C/B3YDzulZcfjnnNYuEfE54MPA34AzKX53WwOvAfaLiLdkZk/DAH9zWsusjf8OPxNY2OJ0bwa2ovg75+gy04ePyj+A3YGnAUHxl9cETp7ga9auc9AY9h0E/lDuv2/d+oHyR5/AR5ocd1y57dPAQJPt03v93o/j/duD4h/QAw3rn0gRviWwn5+bDx8T8yj/e3k+cBNwTPmdPXSCrnVQef7dxrDvRsBfgaXADnXr1wUuK8/zhibHnV1ue2eLex3s9XvuY+o9KMLhBM4DNmyyvet/HvjnnI+p+ij/DrkSuBvYtGHb7uV39uYJuK6/OR9T9sFa+u/wFufaGHik/DvqnLEc0zddtqTxyMyLMvNPWf4SxiIi3hgRF0XEfWWz0usj4vCImDEBJe4KPAO4JDN/Vlf3EMX/hQN4R0REXX1/B7wDuAL4eLnvMJm5fAJqnRSZeWFmntV4X5l5N/DNcnG3xuP83KSueS9F8H0w8HCrnSJiWkS8KyL+r+wq8EhEXBMR746J6fr9OmAT4IeZeWVtZWY+RvF/UQHe2VDjHsArgZ9k5jcaT5iFlRNQq9RS+fv4HMVfzv8pMx9s3KfZnwf+OSd1bEuKf0Rfnpl/rd+QmRdRtAhbY3gBf3NS59bGf4eP4M3AesB/ZeaisVzcLqWakiLiBOCfKZq9/hdwP/Aiii4f8yJir//f3p2HyVXVaRz/vhBkeZCAiIijkgQTlUVACCAYCIOAPgIRhsFB1LSM4DIKAVRADAnLKA8OQnBGGVDS6IOABsOiGFRiWCTKoywzsm/NphEJ2yBJIOE3f/xOkZvbVUkvlU43eT/Pc5+bOufee07q3NO36tRZImJxG5P8x7KfVY+IiIck3Ud2BR9F9jYBOJT80HARsIGk/cmhUfOB2RHxQBvzN9g0PjQsUwYuN7P2kPRu4AxgWkTcUBqsmh23FnA1sC9Lh7AsJH/N/DawM/nho51a1jvgBrLxYldJa8fS7v8fK/tOSZuSQ27eRPZy+GVEPNHmPJr1xK7ASPIX9GckfRjYmqxDt0TE3PoJfs6Z9cv95FQCO0l6Y/ULsaTdgdeTQ8WohLvOmQ2gIVLnWjmi7M/vaeJucLPVjqQOspLPBA6LiAWVuKnAFODfgGltTPadZd9qboX7yYo+hqUVfWzZDy9hG1eOD0nfBY56rfXakDQM+GR5OasS3oHLzazfSh37ITl0+6srOPwksrHtP4FJjftWuXDB+cDhkmZExJVtzGLLehcRiyU9DGxFfjC6u0Q16t0Y4MfAepXTXpZ0akSc3sY8mvVE4778Kzmv4DbVSEk3AAdHxN/K6w78nDPrs4h4WtLxwLeAuyRdQTZKbQEcQA7t/kzjeNc5s4E1hOpcN8pFV7YB7is9ZnvEQ0ptdXQ02XPq8GolL04jH8yHtTnN4WX/XIv4RviGlbA3lf2pwB/ICv56YC/yj8HngcntzeagcAbZA+CaiLi2Eu5yM2uPk8nJ2jua1KVXleFwXyB7iR1T/SBf/n0cOe/FYKp33yQXWxhd4g8CngFOKx/yzAZS4778LDkE5QPk82Br4Fpgd3IS9AY/58z6KSLOIf/2DyN7o5xAzqX4GNBZG2rqOmc2sIZKnWvmyLK/oDeJu4ebrVYkrUeuovIUMKnFUO1F5Djv6nlzyPHfzUyXNL0Wdn1EjO9N1sq+OvZ9zbL/C3Bg5Y/SbEkHk7+WHyvp6xHxUi/SGrQkHUV+ib+HyjA1l5tZe0jaiezVdlaz4Ww1Y8hf0e8Hvtai3i2ge73rIufRaeY3Ta5zUUR0rCAvyyRR9s3q3W3AxMo8IjMlLQauAk4EOnuRjll/Ne5LkT3Z7iiv75R0IPlr+x7lV/M78HPOrN8kfYVcWOBcsnf2PHLVxG8AF0vaLiK+4s+WZgNriNW5ZQ+QhgOHkEPWO3txbTe42WpnI7JSbUJ2We2pTmBOLWwEMBG4Eri9FtdVe91oOR9OcxvUjoPslQEwq/4LQETcUYZVbUH+UbqDIU5So/vwXcBeEfF0JdrlZtZPlaGk99GzX8IbQ1ZGs/x6t37t9Tl0/5VwO2ACOf9MVy2uXg/7Wu82Aa5oMmnvz8kPSGMkDY+IVr9wmrVb43nwUKWxDYCIWCDpWuBfgZ3IId5+zpn1g6Tx5EIlMyPi2ErUrZVG7uMknUd+sXedMxs4Q+n7XN3HyelKLu3pYgkNbnCz1U2jIt0WEe/t6UkR0VkPKw/1ieQXvG7xNfeW/ZgW8aPLvjq2/F5gH3IiyWYaD991V5D2oCdpEnA28Ceyse3J2iEuN7P+W5+l9/LCFr8sXiDpArLx+3slbGZEHNTTRMpwnmWU4ZwTyOE8c1ZwiXuBHUte/1i7zjByEvrFwEO1c8bQpN5FxCuSngfeSNY7N7jZQGk8Q3ryPPBzzqz/9iv7bvMrRcSLkm4BDiSnVWhMW+I6ZzYwhtJzrq6xWMJ/ryCtbjyHm61WIuIF4E5gK0lvGMCkZ5f9B+sRkkaRfwAeYdkvkNeV/dZNzlmbpX8cutqWy1WgTG57NvnrxJ5NGttcbmbtsQj4fovttnLMTeX1XHJo97PALmW10oHSst6Rc16tB9xcWaEUll/vNiUb2/5ODmMwGyg3kI3DoyW9rkl8437t8nPOrC3WLvtNWsQ3wl9ynTMbWEOszlWP2ZkcCntfD3407sYNbrY6+hbwOuBCSd0mR5S0kaQet7r30PXkanq7SzqgktYaZNd3gPNqQ6F+QVb8fSXtXbveZLJb7PURMa/NeR0wkiaTiyT8kezZtrwvwy43s36IiAUR8elmGznHGeR8ap+OiMvKkuzfBjYDzpXU7ZdzSZtJ2rLNWZ1BNoz9i6QdK2mtAzRWGv1u7ZyLycbBDknbVM5ZAzizcd02LzNvtlzlmXYZ+Xf/5GpceT7sS/7i31iR2885s/65seyPlPQP1QhJHwJ2AxYCN5dg1zmzgTVU6lxVY7GE8/uSuIeU2muCpI8AHykv31z275PUWf79VER8CSAiLpS0A7miz4NlDpVHgTeQQ5V2B6aTq4q1RUQskfQpsoV9hqQZJc29yKFTvyV7eVXPeUnSROCXwC8kzSRb38eWPP6NpX8AhpzyfzsVWEJ+QDqqyRC3rkY3YZeb2SpxGvmr3meB/SXNBp4gV0wbTX55OYmce7EtIuJ5SUeQDW9zJF0KPA0cQC7tPoNsxKie85SkI4FLgd9Lupysa3sA7wUeAL7crjya9cKxwM7ASZJ2B24hFxU5kHz+HRERz4Kfc2ZtMAP4Nbki8N3l3pxHzm22Hzl/1AkRMR9c58za4bX4Pbzyf9sA+Cg5F/BFfc2AN29DfgOmkiuLtNq6mtxBCycAAAroSURBVJyzH/Az4MlSieaRH4RPB97VgzTHl2t39CKfWwI/IXtvLCLHip8CrLuCcy6r5PMxcvz4W1f1+76SyyyAOS43b95W/lapj59uEidy1eDryIavl8hGt5vIFU/f1oPrd5Trj+9FnnYDriHnplkA/C9wDLDmcs7Ztfx9mF/y+SDwH8BGq/o99rb6buQXiW8BD5f7cj450fMuLY73c86btz5uwFrAJOB3wPPksO4nS53ap8U5rnPevPVx47X9PfxzJZ1L+vr+qFzIzMzMzMzMzMzM2sBzuJmZmZmZmZmZmbWRG9zMzMzMzMzMzMzayA1uZmZmZmZmZmZmbeQGNzMzMzMzMzMzszZyg5uZmZmZmZmZmVkbucHNzMzMzMzMzMysjdzgZmZmZmZmZmZm1kZucDMzMzNrA0lzJMWqzkc7SRotaaakeZJC0rMrIY3X3PtmZmZmNmxVZ8DMzMysodLw8ijwzohY2OSYLmBzYK2IWDyA2VutSFoTuAJ4B/BD4HGgW3lUju9to9mnIqKzzxkcJMr9SESMWLU5MTMzs8HEDW5mZmY2GL0dmAScsaozshobCWwJXBARR/bg+FOahE0ChgPTgHrvuNvL/pPAen3NpJmZmdlg5AY3MzMzG2yeAQI4UdL3IuKpVZ2h1dRbyv7PPTk4IqbWwyR1kA1u50REV4vzHu1b9szMzMwGL8/hZmZmZoPNi8BpwAbAlJ6cIGl8mWNsaov4rsbQv0pYRzmnQ9Lekm6U9IKkv0maLmnDctz2kn4m6ZkSf5WkEcvJy9qSTpf0sKRFkh6UNEXS61oc/y5JnZIeK8f/VdKPJL2zybGdJc+jJH1R0v9IWiBpTg/fpx0kXS7pyZLWI5K+I2mz2nEBXF9eTilptnx/+6PZHG7V8pS0o6RZkp4rZXC5pLeV40ZJurSU2QJJv5G0bYt01pN0oqTbJf29lOVcSYc2OVaSJkq6uVx7YSmfayV9tJpHcnjz5pX3KCR11q7X1zI+VtI9Jf3HJZ0taYMm57xH0iXlPl9U8nyrpHMkrdWL4jAzM7M2cQ83MzMzG4z+C/gC8BlJ346I+1ZiWgcA+wE/A84DdgU6gJGSTgCuA24Evg9sA+wPbCFpm4h4pcn1fgyMBWYALwMTgKnAjpIOiIhXG5ckfRD4KbAWcDXwAPBW4CDgw5L2jIhbm6QxDRgH/By4Bliyov+kpP2AywGVvD0C7AB8DpggabdKL7RTgBHARLLhbU4Jn8PAGgscX/JwAfn+HwRsI+kA4CbgHuAHZMPXQcCvJI2KiBcaFymNp7OB7YFbgQvJH573BX4kaauI+Fol3X8HTgQeJsvzOWCzkp9/Bi4Dusj3aVI555zK+Y3hsv0p47OB3Uv6V5a8TgLGSXp/Y35DSe8Bfk/2Cr2q5HkDcu69zwNfI+9DMzMzG0gR4c2bN2/evHnzNig2stHg8fLvg8vrn9aO6Srhwyph40vY1BbX7QK6amEd5ZzFwB6V8DWAX5W4p4HDaud9v8RNqIXPKeH3ARtVwtcB5pa4T1TCNyKHzz4FbFm71lbAC8CttfDOcp0ngJG9eF/XL+ksAcbV4o4v1/xlLXy572kP022U1YjlHDMnP5I2TTuW8/4/DZxUi5tc4o5u8b59pRa+DjALeAXYrhI+n1wkYr0m+X3jiu6tNpXxU8Dmtfvy8hI3uRJ+VrP7sZL+Gn0tP2/evHnz5s1b3zcPKTUzM7NBKSJmkA1VB0p6/0pM6pKIaAyfJLLX2g/Lyz9FxMW1439Q9tu1uN5pEfFM5XoLyd5SAIdXjvsksCEwJSLuql4gIu4ke3RtL2nLJmmcGREPL+f/VDcB2Bi4LCJurMWdRTYa7S3p7b245kC4qcn7f1HZP0f3RTW6lY2kjYGPA3+IiDOrB5eyOZ7s9fex2rVepknPwejdnIL9KeNpEfFI5fhXgC+TjYOHNzl+QZO8PhPNe2GamZnZSuYhpWZmZjaYHQfcDJwlaZeIiBWd0Ad/aBLWWCjgj03inij7t7a43vVNwm4ke9JtXwl7X9lv22JutDFl/27grlrcLS3SbuW9ZT+7HhERiyXdQA4h3R4YTIsYLK9sbo+IeoNYs7IZC6wJtJqDrjHH2bsrYRcDXwTulPQTskznRsRzvcg79K+Mu91HEfGQpMeAEZI2jIhnyeGtRwNXSJoB/Br4bUQ82Mu8mpmZWRu5wc3MzMwGrYiYWxoRDgYOIRsX2q1ZI8riHsS1moz+r/WAiFgiaT7wpkrwxmV/xAryt36TsHkrOKdueNn/pUV8I3zDXl53ZetV2ZTGQ1i2bBrv89iytVJ9n48BHiR7kp1QtsWSrgGOi4gHepT7/pVxt/uomEfOVzcceDYibpE0DjiJrCefAJB0L3BKRFzSw7yamZlZG3lIqZmZmQ12J5DD+76hFit9ksPsoPWPicNbhK8Mm9YDJK1JNr48XwluNBhtGxFaznZR/XrknF290UjrzS3iN6sd91rS+D+dvYL3ec/GCRGxJCKmRcS2ZHn+EzCTXGBjlqS1e5l2X8q4231UNMrw1bKKiLkRsR85Z9tu5Cq/m5ILQnygh3k1MzOzNnKDm5mZmQ1qZWjcd4CR5DC/Zhpzpr2tHiHpHQxsz609moSNIxsDb6uE/a4St7I10h1fj5A0DGjMkddstcyh7hayQbZP73NEPBkRP42IQ8ghuVsAW1cOWUIOWW2mP2Xc7T6SNIq8x7vKcNJ6XhdFxM0RcTJwVAme0Ie0zczMrJ/c4GZmZmZDwanAs+SwuWbD7+4he49NkPTqsE1J6wLnDkgOl5osaaNKHtYBvlFeTq8cN538P02RtFP9IpLWkDS+TXm6glzV81BJu9TiJgGjgF9HxGCav60tIuJJck62HSVNLg2My5C0haSR5d9rS9pLZWxq5Zi1gDeUly9WouYDm5R7ra4/ZXy0pM2rxwLfJD+/T6+Ej5PUrAdno4fci03izMzMbCXzHG5mZmY26EXE05K+DpzZIv5lSdOAycBtkmaSn3P2JifZ/3Oz81aSu8nJ9meQQ2EnkL2ifs7S1U+JiPmSDiaHKv5O0nXAnWRvrLeTE+5vDKzT3wxFxAuSDgd+AlxfFgJ4FNgB2IecF+wz/U1nEPsCMJpsuP2EpJvIOdLeQi5YMBY4FHgYWJdceKBL0u+BR8gy2Lsce1VE3F259nXl/Fll8YlFwB0RcXU/y/i3wO2SLiOHj+4LbEsu5FGtB8cB+0iaAzwEvABsBXyI7Pl5fl/eMDMzM+sfN7iZmZnZUHEu8HlyNc1mppC9eY4AjiQbkS4FptJ9BciV6RCy4e8wskHniZKHM+qrrEbEdZLeA3yJbFAZB7xENhDOBi5vV6Yi4kpJuwFfLWkNJ9+j84DTImIgGyUHVEQ8L2kP8r74GDkn2zpko9v95CIJvyqH/x04HtgT2BX4CPB/5CIKnwMurF3+dHLI8v7k/GlrAhcBV5e0+1rGxwAHkvfzCLIn3TTg5IhYWDnuO2TD2s4l/WHA4yX8rIh4pEdvkpmZmbWVap/7zMzMzMxsFZHUCUwERkZE16rNjZmZmfWV53AzMzMzMzMzMzNrIze4mZmZmZmZmZmZtZEb3MzMzMzMzMzMzNrIc7iZmZmZmZmZmZm1kXu4mZmZmZmZmZmZtZEb3MzMzMzMzMzMzNrIDW5mZmZmZmZmZmZt5AY3MzMzMzMzMzOzNnKDm5mZmZmZmZmZWRu5wc3MzMzMzMzMzKyN/h8xfgigtKZeMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if __name__=='__main__':\n",
    "    seed = 1\n",
    "\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "\n",
    "    torch.set_num_threads(1)\n",
    "\n",
    "    env_id = \"PongNoFrameskip-v4\"\n",
    "    envs = [make_env_a2c_atari(env_id, seed, i, log_dir) for i in range(config.num_agents)]\n",
    "    envs = SubprocVecEnv(envs) if config.num_agents > 1 else DummyVecEnv(envs)\n",
    "\n",
    "    obs_shape = envs.observation_space.shape\n",
    "    obs_shape = (obs_shape[0] * 4, *obs_shape[1:])\n",
    "\n",
    "    model = Model(env=envs, config=config)\n",
    "\n",
    "    current_obs = torch.zeros(config.num_agents, *obs_shape,\n",
    "                    device=config.device, dtype=torch.float)\n",
    "\n",
    "    def update_current_obs(obs):\n",
    "        shape_dim0 = envs.observation_space.shape[0]\n",
    "        obs = torch.from_numpy(obs.astype(np.float32)).to(config.device)\n",
    "        current_obs[:, :-shape_dim0] = current_obs[:, shape_dim0:]\n",
    "        current_obs[:, -shape_dim0:] = obs\n",
    "\n",
    "    obs = envs.reset()\n",
    "    update_current_obs(obs)\n",
    "\n",
    "    model.rollouts.observations[0].copy_(current_obs)\n",
    "    \n",
    "    episode_rewards = np.zeros(config.num_agents, dtype=np.float)\n",
    "    final_rewards = np.zeros(config.num_agents, dtype=np.float)\n",
    "\n",
    "    start=timer()\n",
    "\n",
    "    print_step = 1\n",
    "    print_threshold = 10\n",
    "    \n",
    "    for frame_idx in range(1, config.MAX_FRAMES+1):\n",
    "        for step in range(config.rollout):\n",
    "            with torch.no_grad():\n",
    "                values, actions, action_log_prob = model.get_action(model.rollouts.observations[step])\n",
    "            cpu_actions = actions.view(-1).cpu().numpy()\n",
    "    \n",
    "            obs, reward, done, _ = envs.step(cpu_actions)\n",
    "\n",
    "            episode_rewards += reward\n",
    "            masks = 1. - done.astype(np.float32)\n",
    "            final_rewards *= masks\n",
    "            final_rewards += (1. - masks) * episode_rewards\n",
    "            episode_rewards *= masks\n",
    "\n",
    "            rewards = torch.from_numpy(reward.astype(np.float32)).view(-1, 1).to(config.device)\n",
    "            masks = torch.from_numpy(masks).to(config.device).view(-1, 1)\n",
    "\n",
    "            current_obs *= masks.view(-1, 1, 1, 1)\n",
    "            update_current_obs(obs)\n",
    "\n",
    "            model.rollouts.insert(current_obs, actions.view(-1, 1), action_log_prob, values, rewards, masks)\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            next_value = model.get_values(model.rollouts.observations[-1])\n",
    "\n",
    "        model.rollouts.compute_returns(next_value, config.GAMMA)\n",
    "            \n",
    "        value_loss, action_loss, dist_entropy = model.update(model.rollouts)\n",
    "        \n",
    "        model.rollouts.after_update()\n",
    "\n",
    "        if frame_idx % 100 == 0:\n",
    "            try:\n",
    "                clear_output()\n",
    "                end = timer()\n",
    "                total_num_steps = (frame_idx + 1) * config.num_agents * config.rollout\n",
    "                print(\"Updates {}, Num Timesteps {}, FPS {},\\nMean/Median Reward {:.1f}/{:.1f}, Min/Max Reward {:.1f}/{:.1f},\\nEntropy {:.5f}, Value Loss {:.5f}, Policy Loss {:.5f}\".\n",
    "                format(frame_idx, total_num_steps,\n",
    "                       int(total_num_steps / (end - start)),\n",
    "                       np.mean(final_rewards),\n",
    "                       np.median(final_rewards),\n",
    "                       np.min(final_rewards),\n",
    "                       np.max(final_rewards), dist_entropy,\n",
    "                       value_loss, action_loss))\n",
    "                plot(log_dir, \"PongNoFrameskip-v4\", 'PPO', \n",
    "                     config.MAX_FRAMES * config.num_agents * config.rollout)\n",
    "            except IOError:\n",
    "                pass\n",
    "\n",
    "    model.save_w()\n",
    "    envs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_idx\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
